{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-17T06:04:41.086243Z",
     "start_time": "2025-05-17T06:04:41.083775Z"
    }
   },
   "cell_type": "code",
   "source": "model_name = \"unsloth/Qwen3-4B\"",
   "id": "e3dc013d4f7ca2f7",
   "outputs": [],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "from evalscope.run import run_task\n",
    "model_name = \"unsloth/Qwen3-4B\"\n",
    "\n",
    "task_cfg = {\n",
    "    'model': model_name,\n",
    "    'datasets': ['gsm8k', 'arc'],\n",
    "    'limit': 5\n",
    "}\n",
    "\n",
    "run_task(task_cfg=task_cfg)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "from evalscope import TaskConfig, run_task\n",
    "\n",
    "\n",
    "task_cfg = TaskConfig(\n",
    "    model=model_name,\n",
    "    api_url='http://127.0.0.1:8801/v1/chat/completions',\n",
    "    eval_type='service',\n",
    "    datasets=[\n",
    "        'data_collection',\n",
    "    ],\n",
    "    dataset_args={\n",
    "        'data_collection': {\n",
    "            'dataset_id': 'modelscope/EvalScope-Qwen3-Test',\n",
    "            'filters': {'remove_until': '</think>'}  # 过滤掉思考的内容\n",
    "        }\n",
    "    },\n",
    "    eval_batch_size=128,\n",
    "    generation_config={\n",
    "        'max_tokens': 30000,  # 最大生成token数，建议设置为较大值避免输出截断\n",
    "        'temperature': 0.6,  # 采样温度 (qwen 报告推荐值)\n",
    "        'top_p': 0.95,  # top-p采样 (qwen 报告推荐值)\n",
    "        'top_k': 20,  # top-k采样 (qwen 报告推荐值)\n",
    "        'n': 1,  # 每个请求产生的回复数量\n",
    "    },\n",
    "    timeout=60000,  # 超时时间\n",
    "    stream=True,  # 是否使用流式输出\n",
    "    limit=100,  # 设置为100条数据进行测试\n",
    ")\n",
    "\n",
    "run_task(task_cfg=task_cfg)"
   ],
   "id": "5c52edba8b5e795c",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-17T06:06:47.733294Z",
     "start_time": "2025-05-17T06:05:09.967064Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from openai import OpenAI\n",
    "\n",
    "client = OpenAI(\n",
    "    api_key=\"c92a8770-78b8-4e73-8c52-02c1dc036b36\", # 请替换成您的ModelScope SDK Token, 参考： https://modelscope.cn/my/myaccesstoken\n",
    "    base_url=\"https://api-inference.modelscope.cn/v1/\"\n",
    ")\n",
    "\n",
    "\n",
    "response = client.chat.completions.create(\n",
    "    model=\"Qwen/Qwen3-32B\",    # ModelScope model_id\n",
    "    messages=[\n",
    "        {\n",
    "            'role': 'system',\n",
    "            'content': 'You are a helpful assistant.'\n",
    "        },\n",
    "        {\n",
    "            'role': 'user',\n",
    "            'content': '用python写一下快排'\n",
    "        }\n",
    "    ],\n",
    "    stream=True\n",
    ")\n",
    "\n",
    "for chunk in response:\n",
    "    if chunk.choices[0].delta.reasoning_content:\n",
    "        print(chunk.choices[0].delta.reasoning_content, end='', flush=True)\n",
    "    else:\n",
    "        print(chunk.choices[0].delta.content, end='', flush=True)"
   ],
   "id": "346f8401c9d02c95",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "好的，我现在需要用Python来实现快速排序算法。让我先回忆一下快速排序的基本原理。快速排序是一种分治的排序算法，它通过选择一个基准元素，将数组分成两个子数组，一个子数组的元素都比基准小，另一个都比基准大，然后递归地对这两个子数组进行排序。\n",
      "\n",
      "首先，我需要确定如何选择基准元素。常见的做法是选第一个元素、最后一个元素或者中间的元素，或者随机选择。为了简单起见，可能先选第一个元素作为基准。\n",
      "\n",
      "然后，我需要将数组分成两部分。这一步通常通过分区操作来完成。比如，用两个指针，一个从左边开始，一个从右边开始，左边的指针找到比基准大的元素，右边的指针找到比基准小的元素，然后交换它们的位置。直到两个指针相遇。或者，也可以使用荷兰国旗问题的思路，将数组分为小于、等于、大于基准的三个部分。\n",
      "\n",
      "不过，对于初学者来说，可能更倾向于使用递归的方法。比如，选择基准后，将数组分成左右两部分，左边的都小于等于基准，右边的都大于等于基准，然后递归处理左右两部分。\n",
      "\n",
      "那具体怎么实现呢？比如，假设有一个数组arr，选择基准pivot，然后将数组分成三个部分：left（小于pivot）、middle（等于pivot）、right（大于pivot），然后递归排序left和right，最后合并。\n",
      "\n",
      "或者，使用原地分区的方法，这样可以节省空间。不过原地分区的实现可能稍微复杂一些，特别是指针的移动和交换操作容易出错。\n",
      "\n",
      "让我先尝试写一个简单的版本，可能不是原地排序，但更容易理解。例如：\n",
      "\n",
      "def quicksort(arr):\n",
      "    if len(arr) <= 1:\n",
      "        return arr\n",
      "    pivot = arr[0]\n",
      "    left = [x for x in arr[1:] if x <= pivot]\n",
      "    right = [x for x in arr[1:] if x > pivot]\n",
      "    return quicksort(left) + [pivot] + quicksort(right)\n",
      "\n",
      "这个版本的快排是正确的吗？比如，当数组中有重复元素的时候，会不会有问题？比如，当基准元素有多个相等的元素时，应该被分到left还是right？上面的代码中，left收集的是小于等于pivot的元素，而right是大于pivot的。所以，当递归合并的时候，应该没问题。不过，这样的实现方式每次都会把基准元素放在中间，左边的元素都小于等于它，右边的都大于它。然后递归下去。\n",
      "\n",
      "但是，这样的实现并不是原地的，会占用额外的空间，对于大数组来说可能不太高效。不过对于小数组或者教学示例来说，可能已经足够。\n",
      "\n",
      "测试一下这个函数是否正确。比如，输入一个无序数组，比如[3,6,8,10,1,2,1]。第一次pivot是3，left是[1,2,1]，right是[6,8,10]。然后递归处理left和right。最终合并应该得到正确的排序结果。\n",
      "\n",
      "那这个函数的稳定性如何？比如，如果有相同的元素，它们的相对位置会不会改变？比如，在left列表中，元素的顺序可能保持原数组中的顺序，所以对于相同元素来说，它们的相对顺序可能被保留。不过，快速排序本身不是稳定的排序算法，所以这可能不影响正确性。不过这里的实现可能在某些情况下保持了相对顺序，但这不是快速排序的特性所要求的。\n",
      "\n",
      "那这个实现的时间复杂度如何？平均情况下是O(n log n)，最坏情况下是O(n²)，比如当数组已经有序的时候。比如，每次选择第一个元素作为基准，那么对于已排序的数组，每次分区都会导致一边没有元素，另一边是n-1个元素，导致最坏情况。这时候可以考虑随机选择基准或者三数取中法来优化。\n",
      "\n",
      "不过，对于当前的问题，用户可能只需要一个基本的实现。所以先写出这个版本，然后可以再考虑优化。\n",
      "\n",
      "那用户的问题是要用Python写快排，可能希望得到一个正确的实现。所以先写出上面的版本，然后指出可能的优化方向。\n",
      "\n",
      "不过，有没有其他需要注意的地方？比如，递归的终止条件是否正确？当数组长度为0或1的时候，直接返回，没错。那这个函数应该可以正常工作。\n",
      "\n",
      "比如，测试例子：\n",
      "\n",
      "输入：quicksort([5,3,7,6,2,1,4])\n",
      "应该返回 [1,2,3,4,5,6,7]\n",
      "\n",
      "让我们手动模拟一下：\n",
      "\n",
      "第一次pivot是5，left是[3,2,1,4]，right是[7,6]。递归处理left和right。\n",
      "\n",
      "处理left数组[3,2,1,4]，pivot是3，left部分是[2,1], right是[4]. 所以递归处理left的left部分[2,1],得到排序后的数组，然后合并。\n",
      "\n",
      "这个过程应该能正确排序。\n",
      "\n",
      "那这个实现是正确的。不过，对于大数组来说，可能会有栈溢出的问题，因为Python的递归深度有限制。比如，如果数组非常大且处于最坏情况，会导致递归深度超过最大限制。不过，这在实际应用中可能需要处理，但对于示例来说，这个实现已经可以了。\n",
      "\n",
      "另外，用户可能希望看到原地排序的实现。那原地排序的快排如何实现呢？\n",
      "\n",
      "原地排序的快排需要通过交换元素来避免使用额外的空间。这通常涉及一个分区函数，该函数返回基准元素的最终位置，然后递归处理左右子数组。\n",
      "\n",
      "例如，可以这样实现：\n",
      "\n",
      "def quicksort_in_place(arr, low, high):\n",
      "    if low < high:\n",
      "        pi = partition(arr, low, high)\n",
      "        quicksort_in_place(arr, low, pi - 1)\n",
      "        quicksort_in_place(arr, pi + 1, high)\n",
      "\n",
      "def partition(arr, low, high):\n",
      "    pivot = arr[high]\n",
      "    i = low - 1\n",
      "    for j in range(low, high):\n",
      "        if arr[j] <= pivot:\n",
      "            i += 1\n",
      "            arr[i], arr[j] = arr[j], arr[i]\n",
      "    arr[i+1], arr[high] = arr[high], arr[i+1]\n",
      "    return i + 1\n",
      "\n",
      "这里选择最后一个元素作为基准，然后进行分区。初始调用是quicksort_in_place(arr, 0, len(arr)-1)。\n",
      "\n",
      "不过，这个实现需要注意参数传递的问题，因为函数会直接修改原数组。比如，调用后原数组会被修改，而不是返回一个新的数组。这可能与之前的非原地实现不同。\n",
      "\n",
      "那用户可能需要一个完整的例子，比如一个封装好的函数，接受数组并返回排序后的数组。比如：\n",
      "\n",
      "def quicksort(arr):\n",
      "    if len(arr) <= 1:\n",
      "        return arr\n",
      "    # 原地分区逻辑？\n",
      "    # 或者，另一种方式？\n",
      "    # 可能需要重新设计函数结构。\n",
      "\n",
      "或者，可以将原地排序的逻辑封装到函数中，比如：\n",
      "\n",
      "def quicksort(arr):\n",
      "    _quicksort_helper(arr, 0, len(arr)-1)\n",
      "    return arr\n",
      "\n",
      "def _quicksort_helper(arr, low, high):\n",
      "    if low < high:\n",
      "        pi = partition(arr, low, high)\n",
      "        _quicksort_helper(arr, low, pi-1)\n",
      "        _quicksort_helper(arr, pi+1, high)\n",
      "\n",
      "def partition(arr, low, high):\n",
      "    pivot = arr[high]\n",
      "    i = low - 1\n",
      "    for j in range(low, high):\n",
      "        if arr[j] <= pivot:\n",
      "            i += 1\n",
      "            arr[i], arr[j] = arr[j], arr[i]\n",
      "    arr[i+1], arr[high] = arr[high], arr[i+1]\n",
      "    return i+1\n",
      "\n",
      "这样，调用quicksort([3,6,8,10,1,2,1])会返回排序后的数组。这个实现是原地的，空间复杂度是O(log n)的递归栈开销。\n",
      "\n",
      "两种实现方式各有优缺点。第一种实现简洁，但空间复杂度较高；第二种实现更高效，但代码稍微复杂一些。\n",
      "\n",
      "用户的问题只是要求用Python写快排，所以可能需要给出其中一种实现。或者两种都给出，让用户选择。\n",
      "\n",
      "不过，用户可能更希望看到简洁的版本，所以可能先给出第一个实现，然后说明另一种方法。\n",
      "\n",
      "总结一下，可能的回答是提供两种实现方式：一种简洁但空间消耗较大的，另一种是原地排序的递归实现。同时指出它们的优缺点。\n",
      "\n",
      "现在，我需要检查代码是否有错误。比如，在原地排序的partition函数中，是否正确处理了所有情况？\n",
      "\n",
      "比如，假设数组是[5, 3, 7, 6, 2, 1, 4]，并且选择最后一个元素4作为pivot。初始i=-1，遍历j从0到5（假设high是6？或者原数组长度？需要具体例子。）\n",
      "\n",
      "比如，假设数组是[5,3,7,6,2,1,4]，low=0, high=6.\n",
      "\n",
      "pivot是4。遍历j从0到5（因为 high是6，而循环是到high）。\n",
      "\n",
      "当j=0，arr[j]=5>4，不交换。i remains -1.\n",
      "\n",
      "j=1, arr[j]=3<=4. i=0, swap arr[0] and arr[1] →数组变为[3,5,7,6,2,1,4]. 现在i=0.\n",
      "\n",
      "j=2, arr[j]=7>4 → no action.\n",
      "\n",
      "j=3, arr[j]=6>4 → no action.\n",
      "\n",
      "j=4, arr[j]=2<=4 → i=1, swap arr[1] and arr[4] →数组变为[3,2,7,6,5,1,4]. i=1.\n",
      "\n",
      "j=5, arr[j]=1 <=4 →i=2, swap arr[2] and arr[5] →数组变为[3,2,1,6,5,7,4]. i=2.\n",
      "\n",
      "循环结束。然后交换arr[i+1]=arr[3]和arr[high]=arr[6]. 此时数组变为[3,2,1,4,5,7,6]. 返回pi=3.\n",
      "\n",
      "之后递归处理左边的0-2和右边的4-6。这样看起来是正确的。\n",
      "\n",
      "所以，原地排序的partition函数是正确的。\n",
      "\n",
      "因此，两种实现都是正确的。\n",
      "\n",
      "那用户的问题可能希望得到一个正确的Python代码示例。因此，我需要给出这两种方式，并解释它们。\n",
      "\n",
      "现在，回到问题，用户可能希望得到代码示例。所以，我应该写出这两种方式，并说明它们的区别。\n",
      "\n",
      "不过，用户可能只需要一种方式。比如，可能更倾向于简洁的版本，或者原地排序的版本。根据常见的教学示例，可能先给出简洁的版本，因为它更容易理解。\n",
      "\n",
      "综上，我应该写出两种实现方式，并指出它们的优缺点。以下是使用 Python 实现的 **快速排序** 的两种常见方式：一种是简洁但空间占用较高的版本，另一种是更高效的 **原地排序（in-place）** 版本。每种方法都附带了详细的注释和示例，方便理解。\n",
      "\n",
      "---\n",
      "\n",
      "### ✅ 方法一：简洁实现（非原地排序）\n",
      "\n",
      "这种实现方式思路清晰，易于理解，但每次递归都会创建新的子列表，因此在空间使用上不如原地排序高效。\n",
      "\n",
      "```python\n",
      "def quicksort(arr):\n",
      "    if len(arr) <= 1:\n",
      "        return arr\n",
      "    pivot = arr[0]  # 选择第一个元素作为基准\n",
      "    left = [x for x in arr[1:] if x <= pivot]  # 小于等于基准的元素\n",
      "    right = [x for x in arr[1:] if x > pivot]  # 大于基准的元素\n",
      "    return quicksort(left) + [pivot] + quicksort(right)\n",
      "\n",
      "# 示例\n",
      "print(quicksort([3, 6, 8, 10, 1, 2, 1]))  # 输出: [1, 1, 2, 3, 6, 8, 10]\n",
      "```\n",
      "\n",
      "---\n",
      "\n",
      "### ✅ 方法二：原地排序（in-place）实现\n",
      "\n",
      "这种方式在内存中直接对数组进行排序，不会额外创建子列表，效率更高，适合处理较大的数据集。使用了递归和分区（partition）操作。\n",
      "\n",
      "```python\n",
      "def quicksort_in_place(arr):\n",
      "    def _quicksort(arr, low, high):\n",
      "        if low < high:\n",
      "            pi = partition(arr, low, high)  # 获取分区点\n",
      "            _quicksort(arr, low, pi - 1)     # 排序左半部分\n",
      "            _quicksort(arr, pi + 1, high)    # 排序右半部分\n",
      "\n",
      "    def partition(arr, low, high):\n",
      "        pivot = arr[high]  # 选择最后一个元素作为基准\n",
      "        i = low - 1        # i是小于基准的边界\n",
      "        for j in range(low, high):\n",
      "            if arr[j] <= pivot:\n",
      "                i += 1\n",
      "                arr[i], arr[j] = arr[j], arr[i]  # 将小于等于 pivot 的元素移到左边\n",
      "        arr[i + 1], arr[high] = arr[high], arr[i + 1]  # 将 pivot 移到正确位置\n",
      "        return i + 1  # 返回 pivot 的索引\n",
      "\n",
      "    _quicksort(arr, 0, len(arr) - 1)\n",
      "    return arr\n",
      "\n",
      "# 示例\n",
      "print(quicksort_in_place([3, 6, 8, 10, 1, 2, 1]))  # 输出: [1, 1, 2, 3, 6, 8, 10]\n",
      "```\n",
      "\n",
      "---\n",
      "\n",
      "### 🔍 两种方法对比\n",
      "\n",
      "| 特性                  | 简洁版本                  | 原地排序版本             |\n",
      "|-----------------------|---------------------------|--------------------------|\n",
      "| 空间复杂度            | O(n)（每次递归创建新列表） | O(log n)（递归栈）       |\n",
      "| 是否改变原数组        | 否（返回新数组）          | 是（直接修改原数组）     |\n",
      "| 适用场景              | 小数据集、教学示例        | 大数据集、性能要求高     |\n",
      "| 实现难度              | 简单                      | 稍复杂（需理解分区逻辑） |\n",
      "\n",
      "---\n",
      "\n",
      "### 🚀 总结\n",
      "\n",
      "- 如果你追求代码的简洁性和可读性，推荐使用第一种方式。\n",
      "- 如果你关注性能和内存使用效率，推荐使用第二种原地排序的方式。\n",
      "- 快速排序的平均时间复杂度为 **O(n log n)**，最坏情况下为 **O(n²)**。可以通过随机选择基准或三数取中法优化性能，避免最坏情况。\n",
      "\n",
      "希望这些实现和说明对你理解快速排序有所帮助！如果你有进一步的需求，比如添加随机选择基准的版本，也可以继续告诉我。"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-17T06:21:24.606550Z",
     "start_time": "2025-05-17T06:19:11.901661Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from evalscope.run import run_task\n",
    "from evalscope.config import TaskConfig\n",
    "\n",
    "task_cfg = TaskConfig(\n",
    "    model='Qwen/Qwen2.5-0.5B-Instruct',\n",
    "    datasets=['gsm8k', 'arc'],\n",
    "    limit=5\n",
    ")\n",
    "\n",
    "run_task(task_cfg=task_cfg)"
   ],
   "id": "b8f7e1c8f6313901",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-05-17 14:19:11,903 - evalscope - INFO - Args: Task config is provided with TaskConfig type.\n",
      "2025-05-17 14:19:11,916 - evalscope - INFO - Loading model Qwen/Qwen2.5-0.5B-Instruct ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading Model from https://www.modelscope.cn to directory: C:\\Users\\51165\\.cache\\modelscope\\hub\\models\\Qwen\\Qwen2.5-0.5B-Instruct\n",
      "Downloading Model from https://www.modelscope.cn to directory: C:\\Users\\51165\\.cache\\modelscope\\hub\\models\\Qwen\\Qwen2.5-0.5B-Instruct\n",
      "Downloading Model from https://www.modelscope.cn to directory: C:\\Users\\51165\\.cache\\modelscope\\hub\\models\\Qwen\\Qwen2.5-0.5B-Instruct\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-05-17 14:19:15,272 - evalscope - INFO - Updating generation config ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading Model from https://www.modelscope.cn to directory: C:\\Users\\51165\\.cache\\modelscope\\hub\\models\\Qwen\\Qwen2.5-0.5B-Instruct\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-05-17 14:19:17,066 - evalscope - INFO - Updating generation config ...\n",
      "2025-05-17 14:19:17,696 - evalscope - INFO - Dump task config to ./outputs\\20250517_141911\\configs\\task_config_7baafe.yaml\n",
      "2025-05-17 14:19:17,698 - evalscope - INFO - {\n",
      "    \"model\": \"Qwen/Qwen2.5-0.5B-Instruct\",\n",
      "    \"model_id\": \"Qwen2.5-0.5B-Instruct\",\n",
      "    \"model_args\": {\n",
      "        \"revision\": \"master\",\n",
      "        \"precision\": \"torch.float16\"\n",
      "    },\n",
      "    \"model_task\": \"text_generation\",\n",
      "    \"template_type\": null,\n",
      "    \"chat_template\": null,\n",
      "    \"datasets\": [\n",
      "        \"gsm8k\",\n",
      "        \"arc\"\n",
      "    ],\n",
      "    \"dataset_args\": {\n",
      "        \"gsm8k\": {\n",
      "            \"name\": \"gsm8k\",\n",
      "            \"dataset_id\": \"modelscope/gsm8k\",\n",
      "            \"model_adapter\": \"generation\",\n",
      "            \"output_types\": [\n",
      "                \"generation\"\n",
      "            ],\n",
      "            \"subset_list\": [\n",
      "                \"main\"\n",
      "            ],\n",
      "            \"metric_list\": [\n",
      "                \"AverageAccuracy\"\n",
      "            ],\n",
      "            \"few_shot_num\": 4,\n",
      "            \"few_shot_random\": false,\n",
      "            \"train_split\": null,\n",
      "            \"eval_split\": \"test\",\n",
      "            \"prompt_template\": \"Question: {query}\\nLet's think step by step\\nAnswer:\",\n",
      "            \"system_prompt\": null,\n",
      "            \"query_template\": null,\n",
      "            \"pretty_name\": \"GSM8K\",\n",
      "            \"filters\": null,\n",
      "            \"extra_params\": {}\n",
      "        },\n",
      "        \"arc\": {\n",
      "            \"name\": \"arc\",\n",
      "            \"dataset_id\": \"modelscope/ai2_arc\",\n",
      "            \"model_adapter\": \"generation\",\n",
      "            \"output_types\": [\n",
      "                \"multiple_choice_logits\",\n",
      "                \"generation\"\n",
      "            ],\n",
      "            \"subset_list\": [\n",
      "                \"ARC-Easy\",\n",
      "                \"ARC-Challenge\"\n",
      "            ],\n",
      "            \"metric_list\": [\n",
      "                \"AverageAccuracy\"\n",
      "            ],\n",
      "            \"few_shot_num\": 0,\n",
      "            \"few_shot_random\": false,\n",
      "            \"train_split\": \"train\",\n",
      "            \"eval_split\": \"test\",\n",
      "            \"prompt_template\": \"Given the following question and four candidate answers (A, B, C and D), choose the best answer.\\n{query}\\nYour response should end with \\\"The best answer is [the_answer_letter]\\\" where the [the_answer_letter] is one of A, B, C or D.\",\n",
      "            \"system_prompt\": null,\n",
      "            \"query_template\": null,\n",
      "            \"pretty_name\": \"ARC\",\n",
      "            \"filters\": null,\n",
      "            \"extra_params\": {}\n",
      "        }\n",
      "    },\n",
      "    \"dataset_dir\": \"C:\\\\Users\\\\51165\\\\.cache\\\\modelscope\\\\hub\\\\datasets\",\n",
      "    \"dataset_hub\": \"modelscope\",\n",
      "    \"generation_config\": {\n",
      "        \"max_length\": 2048,\n",
      "        \"max_new_tokens\": 512,\n",
      "        \"do_sample\": false,\n",
      "        \"top_k\": 50,\n",
      "        \"top_p\": 1.0,\n",
      "        \"temperature\": 1.0\n",
      "    },\n",
      "    \"eval_type\": \"checkpoint\",\n",
      "    \"eval_backend\": \"Native\",\n",
      "    \"eval_config\": null,\n",
      "    \"stage\": \"all\",\n",
      "    \"limit\": 5,\n",
      "    \"eval_batch_size\": 1,\n",
      "    \"mem_cache\": false,\n",
      "    \"use_cache\": null,\n",
      "    \"work_dir\": \"./outputs\\\\20250517_141911\",\n",
      "    \"outputs\": null,\n",
      "    \"debug\": false,\n",
      "    \"dry_run\": false,\n",
      "    \"seed\": 42,\n",
      "    \"api_url\": null,\n",
      "    \"api_key\": \"EMPTY\",\n",
      "    \"timeout\": null,\n",
      "    \"stream\": false,\n",
      "    \"judge_strategy\": \"auto\",\n",
      "    \"judge_worker_num\": 1,\n",
      "    \"judge_model_args\": {}\n",
      "}\n",
      "2025-05-17 14:19:17,699 - evalscope - INFO - **** Start evaluating on dataset modelscope/gsm8k ****\n",
      "2025-05-17 14:19:17,699 - evalscope - INFO - Loading dataset from hub: modelscope/gsm8k\n",
      "2025-05-17 14:19:17,700 - evalscope - INFO - Loading dataset: dataset_name: modelscope/gsm8k > subsets: ['main']\n",
      "2025-05-17 14:19:23,685 - evalscope - INFO - Use settings: > few_shot_num: 4, > few_shot_split: None, > target_eval_split: test\n",
      "Predicting(main): 100%|██████████| 5/5 [00:44<00:00,  8.92s/it]\n",
      "2025-05-17 14:20:08,296 - evalscope - INFO - Dump predictions to ./outputs\\20250517_141911\\predictions\\Qwen2.5-0.5B-Instruct\\gsm8k_main.jsonl.\n",
      "Reviewing(main): 100%|██████████| 5/5 [00:00<00:00, 2492.16it/s]\n",
      "2025-05-17 14:20:08,304 - evalscope - INFO - Dump report: ./outputs\\20250517_141911\\reports\\Qwen2.5-0.5B-Instruct\\gsm8k.json \n",
      "\n",
      "2025-05-17 14:20:08,307 - evalscope - INFO - Report table: \n",
      "+-----------------------+-----------+-----------------+----------+-------+---------+---------+\n",
      "| Model                 | Dataset   | Metric          | Subset   |   Num |   Score | Cat.0   |\n",
      "+=======================+===========+=================+==========+=======+=========+=========+\n",
      "| Qwen2.5-0.5B-Instruct | gsm8k     | AverageAccuracy | main     |     5 |     0.6 | default |\n",
      "+-----------------------+-----------+-----------------+----------+-------+---------+---------+ \n",
      "\n",
      "2025-05-17 14:20:08,308 - evalscope - INFO - **** Evaluation finished on modelscope/gsm8k ****\n",
      "\n",
      "2025-05-17 14:20:08,308 - evalscope - INFO - **** Start evaluating on dataset modelscope/ai2_arc ****\n",
      "2025-05-17 14:20:08,309 - evalscope - INFO - Loading dataset from hub: modelscope/ai2_arc\n",
      "2025-05-17 14:20:08,309 - evalscope - INFO - Loading dataset: dataset_name: modelscope/ai2_arc > subsets: ['ARC-Easy', 'ARC-Challenge']\n",
      "2025-05-17 14:20:30,446 - evalscope - INFO - Use settings: > few_shot_num: 0, > few_shot_split: train, > target_eval_split: test\n",
      "Predicting(ARC-Easy): 100%|██████████| 5/5 [00:24<00:00,  4.87s/it]\n",
      "2025-05-17 14:20:54,939 - evalscope - INFO - Dump predictions to ./outputs\\20250517_141911\\predictions\\Qwen2.5-0.5B-Instruct\\arc_ARC-Easy.jsonl.\n",
      "Reviewing(ARC-Easy): 100%|██████████| 5/5 [00:00<00:00, 524.42it/s]\n",
      "Predicting(ARC-Challenge): 100%|██████████| 5/5 [00:29<00:00,  5.88s/it]\n",
      "2025-05-17 14:21:24,361 - evalscope - INFO - Dump predictions to ./outputs\\20250517_141911\\predictions\\Qwen2.5-0.5B-Instruct\\arc_ARC-Challenge.jsonl.\n",
      "Reviewing(ARC-Challenge): 100%|██████████| 5/5 [00:00<00:00, 2499.29it/s]\n",
      "2025-05-17 14:21:24,368 - evalscope - INFO - Dump report: ./outputs\\20250517_141911\\reports\\Qwen2.5-0.5B-Instruct\\arc.json \n",
      "\n",
      "2025-05-17 14:21:24,372 - evalscope - INFO - Report table: \n",
      "+-----------------------+-----------+-----------------+---------------+-------+---------+---------+\n",
      "| Model                 | Dataset   | Metric          | Subset        |   Num |   Score | Cat.0   |\n",
      "+=======================+===========+=================+===============+=======+=========+=========+\n",
      "| Qwen2.5-0.5B-Instruct | arc       | AverageAccuracy | ARC-Easy      |     5 |     0.6 | default |\n",
      "+-----------------------+-----------+-----------------+---------------+-------+---------+---------+\n",
      "| Qwen2.5-0.5B-Instruct | arc       | AverageAccuracy | ARC-Challenge |     5 |     0.4 | default |\n",
      "+-----------------------+-----------+-----------------+---------------+-------+---------+---------+\n",
      "| Qwen2.5-0.5B-Instruct | gsm8k     | AverageAccuracy | main          |     5 |     0.6 | default |\n",
      "+-----------------------+-----------+-----------------+---------------+-------+---------+---------+ \n",
      "\n",
      "2025-05-17 14:21:24,372 - evalscope - INFO - **** Evaluation finished on modelscope/ai2_arc ****\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'gsm8k': Report(name='Qwen2.5-0.5B-Instruct_gsm8k', dataset_name='gsm8k', model_name='Qwen2.5-0.5B-Instruct', score=0.6, metrics=[Metric(name='AverageAccuracy', num=5, score=0.6, macro_score=0.6, categories=[Category(name=('default',), num=5, score=0.6, macro_score=0.6, subsets=[Subset(name='main', score=0.6, num=5)])])]),\n",
       " 'arc': Report(name='Qwen2.5-0.5B-Instruct_arc', dataset_name='arc', model_name='Qwen2.5-0.5B-Instruct', score=0.5, metrics=[Metric(name='AverageAccuracy', num=10, score=0.5, macro_score=0.5, categories=[Category(name=('default',), num=10, score=0.5, macro_score=0.5, subsets=[Subset(name='ARC-Easy', score=0.6, num=5), Subset(name='ARC-Challenge', score=0.4, num=5)])])])}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "8f10928ff7041017"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
